#/bin/bash
set -x
Cur_day=`date -d '4 day ago' +%Y%m%d`
#运行时传城市参数进来 $1
city_today=$1"_$Cur_day"
PATH=`pwd`
Local_inputdata_path=$PATH/$1
Local_code_path=$PATH
Local_hadoop_jar=/usr/local/apps/hadoop-2.6.4/share/hadoop/tools/lib/hadoop-streaming-2.6.4.jar
start_time=`date +s`;
<<!
python dumpdata.py $2 $1 house_rent_gov
python dumpdata.py $2 $1 subwaytoborough
python dumpborough.py $1 $2
python dump_pricelist.py $1 $3
!
hadoop fs -test -d /ETL/rent/$city_today
if [ $? -ne 0 ]; then
    hadoop fs -mkdir /ETL/rent/$city_today
else
    hadoop fs -rmr /ETL/rent/$city_today/*
fi
hadoop fs -mkdir /ETL/rent/$city_today
hadoop fs -copyFromLocal $Local_inputdata_path/house_rent_gov /ETL/rent/$city_today
mkdir $Local_inputdata_path/$city_today
mkdir $Local_inputdata_path/$city_today/house_rent_gov
mkdir $Local_inputdata_path/$city_today/house_rent_bad
hadoop fs -rmr /ETL/rent/$city_today/mapred1
hadoop jar $Local_hadoop_jar \
-libjars "./TestMultipleOutputFormat.jar" \
-D mapreduce.job.reduces=0 \
-input /ETL/rent/$city_today/house_rent_gov \
-output /ETL/rent/$city_today/mapred1 \
-outputformat com.zhugefang.hadoop.TestMultipleOutputFormat \
-jobconf suffix.multiple.outputformat.separator="#" \
-mapper '/usr/bin/php borough_filter.php' \
-file "$Local_code_path/borough_filter.php" \
-file "$Local_code_path/rent.php" \
-file "$Local_code_path/rent.php" \
-file "$Local_code_path/function.php" \
-file "$Local_inputdata_path/borough" \
-file "$Local_inputdata_path/borough_price"


hadoop fs -rmr /ETL/rent/$city_today/mapred2
hadoop jar $Local_hadoop_jar \
-libjars "./TestMultipleOutputFormat.jar" \
-D mapreduce.job.reduces=0 \
-input /ETL/rent/$city_today/mapred1/part-*-B \
-output /ETL/rent/$city_today/mapred2 \
-outputformat com.zhugefang.hadoop.TestMultipleOutputFormat \
-jobconf suffix.multiple.outputformat.separator="#" \
-mapper '/usr/bin/php mapper6.php' \
-file "$Local_code_path/mapper6.php" \
-file "$Local_code_path/rent.php" \
-file "$Local_code_path/rent.php" \
-file "$Local_code_path/function.php" 
rm -rf ./$1/$city_today/house_rent_bad/*
hadoop fs -copyToLocal /ETL/rent/$city_today/mapred2* ./$1/$city_today/house_rent_bad
hadoop fs -getmerge /ETL/rent/$city_today/mapred2/part* ./$1/$city_today/house_rent_bad/house_rent_bad
hadoop fs -rmr /ETL/rent/$city_today/mapred3
hadoop jar $Local_hadoop_jar \
-D mapreduce.job.reduces=0 \
-input /ETL/rent/$city_today/mapred1/part-*-N \
-output /ETL/rent/$city_today/mapred3 \
-mapper '/usr/bin/php mapper5.php' \
-file "$Local_code_path/mapper5.php" \
-file "$Local_code_path/function.php"
rm -rf ./$1/$city_today/house_rent_gov/*
hadoop fs -copyToLocal /ETL/rent/$city_today/mapred3/* ./$1/$city_today/house_rent_gov
hadoop fs -getmerge /ETL/rent/$city_today/mapred3/part* ./$1/$city_today/house_rent_gov/house_rent_gov

hadoop fs -rmr /ETL/rent/$city_today/mapred9
hadoop jar $Local_hadoop_jar \
-D mapreduce.job.reduces=0 \
-input /ETL/rent/$city_today/mapred1/part-*-N \
-output /ETL/rent/$city_today/mapred9 \
-mapper '/usr/bin/php mapper9.php' \
-file "$Local_code_path/mapper9.php" \
-file "$Local_code_path/function.php"
rm -rf ./$1/$city_today/gov_json
hadoop fs -copyToLocal /ETL/rent/$city_today/mapred9/* ./$1/$city_today/gov_json
hadoop fs -getmerge /ETL/rent/$city_today/mapred9/part* ./$1/$city_today/gov_json/gov_json
end_time=`date +s`
echo "time spended\n";
echo $(($end_time-$start_time))
